import datetime
from copy import deepcopy
from tqdm import tqdm
import pandas as pd
from jili.tool.state import get_islocal,tick_url,bar_url,data_url
from jili.core import load,save
from jili.tool import convert
import os
from jili.tool.convert import str2datetime
from jili.tool.state import config,telnet
from pymongo import MongoClient as mc
from jili.core.printlog import print
from jili.tool import sortdict
from jili.core.convert import str2datetime
from jili.core import load_localdata as load_localdata0
from jili.core import save_localdata as save_localdata0
import pytz
# 交易所信息与日历
now=datetime.datetime.now()
port0=load_localdata0("mongodb_port")
if port0 is None:
    port0={}
def get_distinct_keys_batched(cu,key,find={}):
    pipeline = []
    if find:
        pipeline.append({"$match": find})
    pipeline =pipeline+ [
        {"$group": {"_id": "$"+key}},  # 按 key 分组
        {"$project": {"_id":0,key: "$_id"}},  # 提取 key 值
    ]
    batch = list(cu.aggregate(pipeline, allowDiskUse=True))
    distinct_keys=[item[key] for item in batch]
    distinct_keys=set(distinct_keys)
    return distinct_keys
def getdb_client(dbname="db",ip=None,auth="ant:ant",port=None):
    global port0
    if ip is None:
        ip=config.mongodb_ip
    if port:
        url = "mongodb://" + auth + "@" + ip + ":" + port + "/" + dbname
        clientk = mc(url)
    else:
        if ip not in port0:
            sysport="27777"
            if telnet(ip,int(sysport)):
                port0[ip] = sysport
                save_localdata0("mongodb_port", port0)
            elif telnet(ip,27017):
                sysport = "27017"
                port0[ip] = sysport
                save_localdata0("mongodb_port", port0)
        else:
            sysport=port0[ip]
        try:
            url="mongodb://"+auth+"@"+ip+":"+sysport+"/" + dbname
            clientk = mc(url)
        except Exception as e:
            # sysport = "27017"
            # port0[ip]=sysport
            # save_localdata0("mongodb_port",port0)
            # url = "mongodb://" + auth + "@" + ip + ":" + sysport + "/" + dbname
            # clientk = mc(url)
            print("error", e)
    return clientk[dbname]
def get_month_dates(datestr,tradeday_region="C"):
    pass
def mongo_index_check_create(cu,indexs,unique_index=None):
    indexs0=[]
    for i in cu.list_indexes():
        v = i.to_dict()
        if v["name"] != "_id_":
            k=list(v["key"])
            indexs0.append(k)
    unique_index1=None
    if unique_index:
        if isinstance(unique_index, str):
            if [unique_index] not in indexs0:
                unique_index1=unique_index
        else:
            if unique_index not in indexs0:
                unique_index1 = unique_index
    indexs1=[]
    if indexs:
        if isinstance(indexs, str):
            if [indexs] not in indexs0:
                indexs1.append(indexs)
        else:
            for key in indexs:
                if isinstance(key,str):
                    if [key] not in indexs0:
                        indexs1.append(key)
                else:
                    if key not in indexs0:
                        indexs1.append(key)
    if indexs1 or unique_index1 is not None:
        cu_create_index(cu, indexs1, unique_index1)
def cu_create_index(cu,indexs,unique_index=None):
    if unique_index:
        if isinstance(unique_index, str):
            cu.create_index([(unique_index, 1)], unique=False, background=True)
        else:
            index0=[]
            for j in unique_index:
                if isinstance(j, str):
                    index0.append((j,1))
                else:
                    index0.append(j)
            cu.create_index(index0, unique=False, background=True)
    for i in indexs:
        if isinstance(i, str):
            cu.create_index([(i, 1)], unique=False, background=True)
        else:
            index0=[]
            for j in i:
                if isinstance(j, str):
                    index0.append((j,1))
                else:
                    index0.append(j)
            cu.create_index(index0, unique=False, background=True)
def save_mongo_bykey(cu,data,keys,have,iswinddate=True):
    n=0
    for i in data:
        key = "k"
        for k in keys:
            tmp = str(i[k])
            tmp = tmp.replace(" ", "")
            key = key + "-" + tmp
        if key not in have:
            n = n + 1
            # if isinstance(i["date"],datetime.datetime):
            #     i["date"]=i["date"].strftime("%Y-%m-%d")
            insert_one(cu, i)
    print("总计，存在，新增", len(data), len(have), n)
def getmongodb(cu,aa,keys,backdays=1,finds={},datestr="%Y%m%d"):
    maxd=None
    have=[]
    for i in cu.find(finds).sort([(aa,-1)]):
        if not maxd:
            maxd=i[aa]
            break
    if maxd:
        t=str2datetime(maxd)-datetime.timedelta(days=backdays)
        maxd=t.strftime(datestr)
        find0={aa:{"$gte":maxd}}
        find0.update(finds)
        #print(find0)
        n=0
        for i in cu.find(find0):
            key="k"
            for k in keys:
                if k in i.keys():
                    tmp=str(i[k])
                    tmp=tmp.replace(" ","")
                    key=key+"-"+tmp
            if key not in have:
                have.append(key)
            n=n+1
        print("getmongodb",n,maxd)
    return maxd,have
def insert_one(cu, t,updteflag=False,id=None):
    try:
        if "uptime" in t.keys():
            if pd.isna(t["uptime"]):
                t["uptime"]=datetime.datetime.now()
        if updteflag:
            t0=deepcopy(t)
            t=t0
            if isinstance(id,str):
                if id in t.keys():
                    id0=t.pop(id)
                    if "_id" in t.keys():
                        t.pop("_id")
                    cu.update_one({id:id0},{"$set":t})
            elif isinstance(id,list):
                keys={}
                for i in id:
                    keys[i]=t.pop(i)
                if "_id" in t.keys():
                    del t["_id"]
                cu.update_one(keys, {"$set": t})
            else:
                if "_id" in t.keys():
                    id0 = t.pop("_id")
                    cu.update_one({"_id": id0}, {"$set": t})
        else:
            cu.insert_one(t)
    except Exception as e:
        print("error",e,updteflag,id,t)
def insert_one_nolog(cu, t,updteflag=False,id=None):
    try:
        if updteflag:
            t0 = deepcopy(t)
            t = t0
            if isinstance(id, str):
                if id in t.keys():
                    id0 = t.pop(id)
                    if "_id" in t.keys():
                        t.pop("_id")
                    cu.update_one({id: id0}, {"$set": t})
            elif isinstance(id, list):
                keys = {}
                for i in id:
                    keys[i] = t.pop(i)
                if "_id" in t.keys():
                    del t["_id"]
                cu.update_one(keys, {"$set": t})
            else:
                if "_id" in t.keys():
                    id0 = t.pop("_id")
                    cu.update_one({"_id": id0}, {"$set": t})
        else:
            cu.insert_one(t)
    except Exception as e:
        pass

class mssqldb:
    def __init__(self,dbname,ip,auth):
        import pymssql
        self.ip=ip
        self.user=auth.split(":")[0]
        self.password = auth.split(":")[1]
        self.db = pymssql.connect(self.ip, self.user, self.password, dbname)
        self.cursor = self.db.cursor(as_dict=True)
    def insert(self,table,d):
        try:
            sql=self.generate_sqltable(table,d)
            self.cursor.execute(sql)
            self.db.commit()
        except Exception as e:
            print("mssqldb insert err",e)
    def generate_sqltable(self,table,data):
        cols = ""
        val_cols=""
        for k,v in data.items():
            if val_cols=="":
                if k in ["open","close","date","type","source"]:
                    k="["+k+"]"
                cols=k
                if pd.isna(v):
                    val_cols="null"
                elif isinstance(v,str):
                    val_cols = "'" + v + "'"
                elif isinstance(v,datetime.datetime):
                    val_cols = "'" + v.strftime("%Y%m%d") + "'"
                else:
                    val_cols = str(v)
            else:
                if k in ["open", "close", "date", "type","source"]:
                    k = "[" + k + "]"
                cols=cols+","+k
                if pd.isna(v):
                    val_cols = val_cols +","+"null"
                elif isinstance(v, str):
                    val_cols = val_cols+",'"+v+"'"
                elif isinstance(v,datetime.datetime):
                    val_cols = val_cols + ",'" + v.strftime("%Y%m%d") + "'"
                else:
                    val_cols = val_cols +","+str(v)

        sql = '''INSERT INTO %s (%s) VALUES(%s);'''%(table,cols, val_cols)
        return sql
    def insert_many(self,table,data_list):
        try:
            sql = self.generate_sql(table,data_list)
            self.cursor.executemany(sql, data_list)
            self.db.commit()
        except Exception as e:
            print("mssqldb insert err",e)
    def generate_sql(self,table,data_list):
        data = data_list[0]
        cols = ", ".join('`{}`'.format(k) for k in data.keys())
        val_cols = ', '.join('%({})s'.format(k) for k in data.keys())
        sql = """
        INSERT INTO table(%s) VALUES(%s)
        """ % (cols, val_cols)
        return sql
    def close(self):
        self.cursor.close()
        self.db.close()
trade_list={}
def get_calender_day(start,end=None):
    if end is None:
        end=datetime.datetime.now()
        end=datetime.datetime(end.year,end.month,end.day)
    else:
        end=str2datetime(end)
    start=str2datetime(start)
    d0=start
    while d0<=end:
        yield d0
        d0=d0+datetime.timedelta(days=1)
def get_calender_hks(start=None,end=None,ip=None):
    return get_calender(start, end,region="HKS",ip=ip)
def get_calender_shn(start=None,end=None,ip=None):
    return get_calender(start, end, region="SHN", ip=ip)
def get_calender_a(start=None,end=None,ip=None):
    return get_calender(start, end, region="C", ip=ip)
def get_calender_hk(start=None,end=None,ip=None):
    return get_calender(start, end, region="H", ip=ip)
def get_calender(start=None,end=None,region="C",ip=None):
    global trade_list
    if region not in trade_list.keys():
        trade_list[region]=[]
        for i in tradecal.data[region].values():
            if i["istrade"]:
                trade_list[region].append(i["datetime"])
        # if ip   is None:
        #     ip = config.mongodb_ip
        # db = getdb_client(ip=ip)
        # cu = db["tradeset_calender"]
        # trade_list[region] = cu.find({"region": region, "istrade": True}, {"datetime":1}).sort([('datetime', -1)]).distinct(
        #     "datetime")
        # db.client.close()
    l = []
    if end:
        end = convert.str2datetime(end)
    else:
        end = datetime.datetime.now()
    if start:
        start1 = convert.str2datetime(start)
        for i in trade_list[region]:
            if i >= start1 and i <= end:
                l.append(i)
    else:
        for i in trade_list[region]:
            if i <= end:
                l.append(i)
    return l
class TradeCalender:
    def __init__(self, startdate=None,url_pkl=os.path.join(data_url,"tcl.pkl"),ip=None):
        if startdate:
            self.startdate = str2datetime(startdate)
        else:
            self.startdate=None
        self.day = self.startdate
        self.tday = self.startdate
        self.ist = False
        self.data = {}
        self.trade_data={}
        if ip   is None:
            self.ip = config.mongodb_ip
        else:
            self.ip=ip
        t=load(url_pkl)
        if t:
            self.data=t
        self.db="tradeset_calender"
        if startdate:#重新从数据库加载
            self.loaddb(startdate=startdate,ip=ip)
        else:
            self.build_trade_data()
    def build_trade_data(self):
        for k,v in self.data.items():
            t=[]
            vv=sortdict(v,True,by="key")
            for d,i in vv.items():
                if i["istrade"]:
                    t.append(d)
            self.trade_data[k] =t

    def reloaddb(self,startdate,ip=None):
        self.data = {}
        self.loaddb(startdate=startdate,ip=ip)
    def loaddb(self,startdate,ip=None):
        if startdate:
            self.startdate=str2datetime(startdate)
            find={"datetime": {"$gte": self.startdate}}
        else:
            find={}
        if ip is None:
            ip=self.ip
        db = getdb_client(ip=ip)
        cur = db[self.db].find(find).batch_size(5000)
        for i in cur:
            del i["_id"]
            region = i["region"]
            dt1 = i["datetime"]
            if region not in self.data.keys():
                self.data[region] = {}
            self.data[region][dt1] = i
        db.client.close()
        if self.data:
            self.savepkl()
            self.build_trade_data()
    def savepkl(self):
        save(self.data,os.path.join(data_url,"tcl.pkl"))
    #以上是初始化  与存储函数   以下是  功能性函数
    def getday_db(self,day,region="C"):
        db = getdb_client(ip=self.ip)
        day0 = str2datetime(day)
        info= db[self.db].find_one({"datetime": day0,"region":region})
        db.client.close()
        if info:
            del info["_id"]
            dt1 = info["datetime"]
            region = info["region"]
            if region not in self.data.keys():
                self.data[region] = {}
            self.data[region][dt1] = info
            if day0>=datetime.datetime(2010,1,1):
                t=list(self.data[region].keys())
                self.loaddb(startdate=max(t))
            return info
        else:
            return {}
    def getdayinfo(self,day,region="C"):
        day=str2datetime(day)
        day1=datetime.datetime(day.year,day.month,day.day)
        if region in self.data.keys():
            if day1 in self.data[region].keys():
                return self.data[region][day1]
        return self.getday_db(day,region="C")
    def gettradeday(self, day, diff=0,region="C"):
        day=str2datetime(day)
        day=datetime.datetime(day.year,day.month,day.day)
        tday0 = day + datetime.timedelta(days=diff)
        if tday0 != self.day:
            info = self.getdayinfo(tday0,region=region)
            ist0=info["istrade"]
            while (not ist0):
                tday0 = tday0 + datetime.timedelta(days=1)
                info = self.getdayinfo(tday0,region=region)
                ist0=info["istrade"]
            self.tday = tday0
        return self.tday
    def gettradedays(self, start=None,end=None, diff=-1,region="C"):
        ds=[]
        if start is None and end is None:
            if region=="A":
                us = pytz.country_timezones('us')
                tz = pytz.timezone(us[0])
                now = datetime.datetime.now(tz)
            else:
                now = datetime.datetime.now()
            end=datetime.datetime(now.year,now.month,now.day)
        if start is None:
            start=end+datetime.timedelta(days=-abs(diff))
        if end is None:
            end=start+datetime.timedelta(days=abs(diff))
        for d in get_calender(start,end,region):
            r=self.istradeday(d,region=region)
            if r:
                ds.append(d)
        return ds


    def gettradeday_monthend(self,day):
        day = str2datetime(day)
        day = datetime.datetime(day.year, day.month, day.day)
        if day.month==12:
            day1=datetime.datetime(day.year+1, 1, 1)
        else:
            day1 = datetime.datetime(day.year, day.month+1, 1)
        return self.getpretradeday(day1)
    def getnexttradeday(self,day,region="C"):
        day=str2datetime(day)
        day = datetime.datetime(day.year, day.month, day.day)
        tday0 = day + datetime.timedelta(days=1)
        if tday0 != self.day:
            info = self.getdayinfo(tday0,region=region)
            ist0=info["istrade"]
            while (not ist0):
                tday0 = tday0 + datetime.timedelta(days=1)
                info = self.getdayinfo(tday0,region=region)
                ist0=info["istrade"]
            self.tday = tday0
        return self.tday
    def getpretradeday(self,day,region="C"):
        day=str2datetime(day)
        day = datetime.datetime(day.year, day.month, day.day)
        tday0 = day + datetime.timedelta(days=-1)
        if tday0 != self.day:
            info = self.getdayinfo(tday0,region=region)
            if info:
                ist0=info["istrade"]
                while (not ist0):
                    tday0 = tday0 + datetime.timedelta(days=-1)
                    info = self.getdayinfo(tday0,region=region)
                    if info:
                        ist0=info["istrade"]
                    else:
                        print("error","日历超出范围",tday0)
            else:
                print("error","日历超出范围",tday0)
            self.tday = tday0
        return self.tday
    def get_diff_tradeday(self,day,diff,region="C"):
        rstday=str2datetime(day)
        rstday = datetime.datetime(rstday.year, rstday.month, rstday.day)
        if diff>=0:
            n=0
            tday0=rstday
            while n<diff:
                tday0 = tday0 + datetime.timedelta(days=1)
                if self.istradeday(tday0,region=region):
                    n=n+1
            rstday=tday0
        else:
            n = 0
            tday0 = rstday
            while n > diff:
                tday0 = tday0 - datetime.timedelta(days=1)
                if self.istradeday(tday0,region=region):
                    n = n - 1
            rstday = tday0
        return rstday
    def gettradeday_bytimekey(self,timekey,region="C"):
        timekey = str2datetime(timekey)
        d=datetime.datetime(timekey.year,timekey.month,timekey.day)
        if timekey.hour>=18:
            d=self.getnexttradeday(d)
        elif timekey.hour<8:
            if not self.istradeday(d,region=region):
                d=self.gettradeday(d)
        return d
    def istradeday(self, day, diff=0,region="C"):
        day=str2datetime(day)
        day = datetime.datetime(day.year, day.month, day.day)
        tday0 = day + datetime.timedelta(days=diff)
        info = self.getdayinfo(tday0,region=region)
        ist0=info["istrade"]
        return ist0
    def istradenight(self, day, diff=0):
        day=str2datetime(day)
        day = datetime.datetime(day.year, day.month, day.day)
        tday0 = day + datetime.timedelta(days=diff)
        info = self.getdayinfo(tday0,region="C")
        ist0=info["havenighttrade"]
        return ist0
    def count_tradedays(self,start,end,region="C"):
        d0=self.gettradeday(start)
        d1=self.gettradeday(end)
        if region in self.trade_data.keys():
            t=self.trade_data[region]
            n0=t.index(d0)
            n1 = t.index(d1)
            if n0<0:
                return None
            if n1<0:
                return None
            return n1-n0+1
        else:
            return None
    def isweek_end(self,d,region="C"):
        d0 = str2datetime(d)
        d1 = self.getnexttradeday(d0,region)
        if d1.weekday()<d0.weekday():
            return True
        else:
            return False
    def isweek_start(self,d,region="C"):
        d0 = str2datetime(d)
        d1 = self.getpretradeday(d0, region)
        if d1.weekday() > d0.weekday():
            return True
        else:
            return False

tradecal=TradeCalender()
def gettrade_day(timekey):
    if timekey.hour>=16:
        return tradecal.getnexttradeday(timekey)
    else:
        return datetime.datetime(timekey.year,timekey.month,timekey.day)
def getTradetime(date):
    if isinstance(date,datetime.datetime):
        date=date.date()
    elif isinstance(date,str):
        date=str2datetime(date)
    predate=tradecal.getpretradeday(date)
    if tradecal.istradenight(predate):
        t1=datetime.datetime(predate.year,predate.month,predate.day,20,59)
        t2=datetime.datetime(date.year,date.month,date.day,3,30)
        t3=datetime.datetime(date.year,date.month,date.day,8,59)
        t4=datetime.datetime(date.year,date.month,date.day,15,1)
        return (t1,t2,t3,t4)  
    else:
        dtime2=datetime.datetime(date.year,date.month,date.day,8,59)
        dtime3=datetime.datetime(date.year,date.month,date.day,15,1)
        return (dtime2,dtime3) 
"""
1)数据校验：时间合法，价格在涨跌停内，非 ，0，nan,null
    交易时间：
    (一)大连、上海、郑州交易所
        集合竞价申报时间：08：55—08：59
        集合竞价撮合时间：08：59—09：00
        正常开盘交易时间：09：00－11：30 （小节休息10：15－10：30）13：30－15：00
        客户下单时间为集合竞价时间和正常交易时间。在8：59—9：00竞价结束时间和交易所小节休息时间（上午10:15-10:30）下单，交易系统将不接受指令，并视之为废单。（时间以交易所时钟报时为准
        (二)上期所夜盘
        集合竞价申报时间：20：55—20：59
        集合竞价撮合时间：20：59—21：00
        正常开盘交易时间：21：00－02：30 （黄金、白银）
        21：00－01：00 （铜、铝、铅、锌、镍、锡）
        21：00－23：00（螺纹钢、热轧卷板、石油沥青、天然橡胶）
        提示：法定节假日的前一日没有夜盘交易。
        （三）大商所夜盘
        集合竞价申报时间：20：55—20：59
        集合竞价撮合时间：20：59—21：00
        正常开盘交易时间：21：00—23：30 （豆一、豆二、豆油、豆粕、焦煤、焦炭、棕榈油、铁矿石）
        提示：法定节假日的前一日没有夜盘交易。
        （四）郑商所夜盘
        集合竞价申报时间：20：55—20：59
        集合竞价撮合时间：20：59—21：00
        正常开盘交易时间：21：00－23：30 （白糖、棉花、菜粕、甲醇、PTA、菜籽油、玻璃、动力煤）
        提示：法定节假日的前一日没有夜盘交易。
        (五)中金所
        股指:集合竞价时间：9：25—9：30
        正常开盘交易时间：9：30-11：30（第一节）；13：00-15：00（第二节）
        国债：
        集合竞价时间：9：10-9：15
        正常开盘交易时间：9：15-11：30（第一节）；13：00-15：15（第二节）
        最后交易日交易时间：9：15-11：30
    涨跌停：行情中有此指标
    0与nan,null  
    timekey定义：除郑商所毫秒时间不正常外；其他三个交易所正常；TradeDay,ActionDay对于夜盘，郑商所只有实际天无交易日；大连无实际天数概念只有交易日；上交所比较完善
    当前不考虑郑商所品种
2)主力合约获取与聚合；连续合约处理
    
3)技术指标计算；K线清晰
    k线：秒与分钟级别的可以考虑使用pandas的resample ohlc处理
    k线：日k线，考虑使用sina的，雅虎的；后续考虑wind
    小时级别的k线考虑使用算法自己生成；另者所有k线均考虑自己生成确保准确
"""

"""
除郑州市场的原始数据，包含结算数据
"""
alias_name={}
def addaliasname(alias):
    for k,v in alias.items():
        for i in v:
            if i not in alias_name.keys():
                alias_name[i]=k
alias_tick=dict(
    obj=["objid","InstrumentID"]
    ,starttime=["start","s","from","fromtime"]
    ,endtime=["end","e","to","totime"]
)
addaliasname(alias_tick)
def originalname(argkv):
    for k,v in argkv.items():
        if k in alias_name.keys():
            name=alias_name[k]
            argkv[name]=v
def arg2argkv(arg,argkv,argkey):
    n=len(arg)
    for i in range(n):
        key=argkey[i]
        argkv[key]=arg[i]

def getdbone(cu,find={},field={},sort={},db="db",ip=None):
    if ip is None:
        ip=config.mongodb_ip
    db=getdb_client(db=db,ip=ip)
    collection=db[cu]
    fienld0={}
    sort0=[]
    if isinstance(field,list):
        for i in field:
            fienld0[i]=1
    else:
        fienld0=field
    if isinstance(sort,list):
        for i in sort:
            sort0.append((i,1))
    else:
        for k,v in sort.items():
            sort0.append((k,v))
    if fienld0:
        if sort0:
            cur=collection.find_one(find,fienld0).sort(sort0)
        else:
            cur=collection.find_one(find,fienld0)
    else:
        if sort0:
            cur=collection.find_one(find).sort(sort0)
        else:
            cur=collection.find_one(find)
    return cur
def getdb(cu,find={},field={},sort={},db="db",ip=None,datatype=""):
    if ip   is None:
        ip = config.mongodb_ip
    db = getdb_client(db=db, ip=ip)
    collection=db[cu]
    rst=[]
    fienld0={}
    sort0=[]
    if isinstance(field,list):
        for i in field:
            fienld0[i]=1
    else:
        fienld0=field
    if isinstance(sort,list):
        for i in sort:
            sort0.append((i,1))
    else:
        for k,v in sort.items():
            sort0.append((k,v))
    if fienld0:
        if sort0:
            cur=collection.find(find,fienld0).sort(sort0)
        else:
            cur=collection.find(find,fienld0)
    else:
        if sort0:
            cur=collection.find(find).sort(sort0)
        else:
            cur=collection.find(find)
    for i in cur:
        rst.append(i)
    if datatype in ["DF","df","DataFrame","dataframe"]:
        rst=pd.DataFrame(rst)
        rst.index=rst["timekey"]
    db.client.close()
    return rst
def getkline1d_index(obj,datatype="df"):
    a=getdb("kline1dindex", find={"obj" : obj}, field={}, sort={"timekey":1}, db='kline1d', datatype=datatype)
    return a
def getkline1d_indexwi(obj,datatype="df"):
    a=getdb("kline1dindexwi", find={"obj" : obj}, field={}, sort={"timekey":1}, db='kline1d',datatype=datatype)
    return a
def getkline1d(obj,datatype="df"):
    a=getdb("kline1d_f", find={"obj" : obj}, field={}, sort={"timekey":1}, db='kline1d', datatype=datatype)
    return a
def getxhprice(obj,datatype="df"):
    a=getdb("kline1d_xh", find={"objid" : obj}, field={}, sort={"timekey":1}, db='kline1d', datatype=datatype)
    return a

def getkline1d_adj(obj,adjtype=0):
    #a=db.getkline1d("rb1805")
    b=getkline1d_index("rb")
    b1=b.drop_duplicates(["objid"],keep="last",inplace=False)
    b2=b.drop_duplicates(["objid"],keep="first",inplace=False)
    b11=b1.to_dict("index")
    b22={}
    for k,v in b11.items():
        i=getdbone("kline1d_f",find={"obj":v["objid"],"timekey":{"$gt":k}},db="kline1d")
        b22[i["timekey"]]=i
    adj={}
    b21=b2.to_dict("index")
    for k,v in b22.items():
        if k in b21:
            v1=b21[k]
            t={}
            t["percent0"]=v1["close"]/v["close"]
            t["point0"]=v1["close"]-v["close"]
            adj[v["obj"]]=t
    adjd=pd.DataFrame(adj).T
    adjd.sort_index(ascending=False,inplace=True)
    adj=adjd.to_dict("index")
    adj0=1
    adj1=0
    for  k,v in adj.items():
        adj0=adj0*v["percent0"]
        adj1=adj1+v["point0"]
        adj[k]["percent"]=adj0
        adj[k]["point"]=adj1
    if adjtype==0:
        for k,v in adj.items():
            b.loc[b[b["objid"]==k].index,"close"]=b[b["objid"]==k]["close"]+v["point"]
            b.loc[b[b["objid"]==k].index,"open"]=b[b["objid"]==k]["open"]+v["point"]
            b.loc[b[b["objid"]==k].index,"high"]=b[b["objid"]==k]["high"]+v["point"]
            b.loc[b[b["objid"]==k].index,"low"]=b[b["objid"]==k]["low"]+v["point"]
            b.loc[b[b["objid"]==k].index,"vwap"]=b[b["objid"]==k]["vwap"]+v["point"]
    else:
        for k,v in adj.items():
            b.loc[b[b["objid"]==k].index,"close"]=b[b["objid"]==k]["close"]*v["percent"]
            b.loc[b[b["objid"]==k].index,"open"]=b[b["objid"]==k]["open"]*v["percent"]
            b.loc[b[b["objid"]==k].index,"high"]=b[b["objid"]==k]["high"]*v["percent"]
            b.loc[b[b["objid"]==k].index,"low"]=b[b["objid"]==k]["low"]*v["percent"]
            b.loc[b[b["objid"]==k].index,"vwap"]=b[b["objid"]==k]["vwap"]*v["percent"]
    return b
def gettickbydate(obj,date,islocal=get_islocal()):
    date=getTradetime(date)
    start=date[0]
    end=date[-1]
    return getdb(obj,find={"timekey":{"$gte":start,"$lte":end}},sort={"timekey":1},db="tick_f",islocal=islocal,datatype="dict")
def gettick_dfw(obj,date,datetype="df"):
    name=""
    if isinstance(date,str):
        name="~/tick/"+obj+"_"+date+".csv"
    elif isinstance(date,datetime.date):
        name="~/tick/"+obj+"_"+date.strftime("%Y%m%d")+".csv"
    csv=pd.read_csv(name,encoding="gbk")
    csv.rename(columns={'大富翁数据中心_市场代码':"marketcode", '合约代码':"obj", '时间':"timekey", '最新':"LastPrice", '持仓':"position", '增仓':"deltaposition", '成交额':"amont" ,'成交量':"volume", '开仓':"openvolume",'平仓':"closevolume", '成交类型':"dealfeatures", '方向':"bsflag", '买一价':"BidPrice1", '卖一价':"AskPrice1", '买一量':"BidVolume1", '卖一量':"AskVolume1"}, inplace = True)
    csv["timekey"]=csv["timekey"].apply(str2datetime)
    csv["AveragePrice"]=0
    if datetype=="list":
        return csv.to_dict("records")
    elif datetype=="dict":
        csv.index=csv["timekey"]
        return csv.to_dict("index")
    else:
        return csv
#from functools import partial
class tick_dataset_pkl:
    """tick类型数据的迭代器，使用pkl文件源方式，使用for或是next遍历"""
    def __init__(self,obj,start_date=None,end_date=None,url=tick_url):
        """

        Args:
            obj: str  tick数据的代码
            start_date:str tick开始日期：20200101格式，不带中间分隔符；暂时不支持其他格式
            end_date: 同上
        """
        self.tick_url = os.path.join(url, obj)
        self.fs=[]
        fs=os.listdir(self.tick_url)
        fs.sort()
        if start_date:
            self.start_date=start_date
        else:
            self.start_date =fs[0].split("_")[1].split(".")[0]
        if end_date:
            self.end_date=end_date
        else:
            self.end_date =fs[-1].split("_")[1].split(".")[0]
        self.obj=obj
        self.batchdata=[]
        for i in fs:
            date=i.split("_")[1].split(".")[0]
            if (date >=self.start_date) and (date <=self.end_date):
                self.fs.append(i)

    def __iter__(self):
        return self
    def __next__(self):
        """

        Returns:tick数据条目；dict
        t=load(tickurl)
        """
        while True:
            if self.batchdata:
                t= self.batchdata.pop(0)
                #t["timekey"]=t["timekey"].to_pydatetime()
                #t["OpenInterest"]=t["position"]
                #t["AveragePrice"] = 0
                return t
            elif self.fs:
                i=self.fs.pop(0)
                obj_url=os.path.join(self.tick_url, i)
                rst=load(obj_url)
                if rst:
                    self.batchdata=rst
                    if self.batchdata:
                        t = self.batchdata.pop(0)
                        #t["timekey"] = t["timekey"].to_pydatetime()
                        #t["OpenInterest"] = t["position"]
                        #t["AveragePrice"] = 0
                        return t
            else:
                raise StopIteration
def gettick_pkl(obj,date,datetype="df",url=tick_url):
    name=""
    if isinstance(date,str):
        name=obj+"_"+date+".pkl"
    elif isinstance(date,datetime.date):
        name=obj+"_"+date.strftime("%Y%m%d")+".pkl"
    tickurl=os.path.join(os.path.abspath(url),obj,name)
    print(tickurl)
    if os.path.exists(tickurl):
        t=load(tickurl)
        if datetype=="list":
            return t
        elif datetype=="dict":
            t=pd.DataFrame(t)
            t.index=t["timekey"]
            return t.to_dict("index")
        else:
            t = pd.DataFrame(t)
            t.index = t["timekey"]
            return t
    else:
        return None
def getkline1m_rb(obj,startdate="",enddate="",datatype="df"):
    if startdate!="":
        date=getTradetime(startdate)
        startdate=date[0]
    if enddate!="":
        date=getTradetime(startdate)
        enddate=date[-1]
    print(startdate,enddate)
    if startdate=="":
        a=getdb("kline1m_rb", find={"obj" : obj}, field={}, sort={"timekey":1}, db='kline1d', islocal=get_islocal(), datatype=datatype)
    elif enddate=="":
        a=getdb("kline1m_rb", find={"obj" : obj,"timekey":{"$gte":startdate}}, field={}, sort={"timekey":1}, db='kline1d', islocal=get_islocal(), datatype=datatype)
    else:
        a=getdb("kline1m_rb", find={"obj" : obj,"timekey":{"$gte":startdate,"$lte":enddate}}, field={}, sort={"timekey":1}, db='kline1d', islocal=get_islocal(), datatype=datatype)
    return a
def barengine(obj,call,startdate="",enddate="",period=1,callpre=None,file="pkl",url=bar_url):
    if file!="pkl":
        if period!=1:
            a=getkline1m_rb(obj,startdate,enddate,"list")
            from jili.price import kline
            k0=kline.klinenm(obj,period)
            for i in a:
                k0.onbar(i)
            k=pd.DataFrame(k0.bar)
            k.index=k["timekey"]
        else:
            k=getkline1m_rb(obj,startdate,enddate)
        from jili.calc import sigma
        k=sigma.sigma3(k)
        if callpre:
            callpre(k)
        k=k.to_dict("index")
        for k, v in k.items():
            call(k, v)
    else:
        from jili.calc import chan
        f_t = url + obj+"_k1m.pkl"
        tl = load(f_t)
        if period!=1:
            from jili.price import kline
            k0 = kline.klinenm(obj, period)
            for i in tl:
                k0.onbar(i)
        tl=k0.bar
        if callpre:
            callpre(tl)
        for t in tl:
            call(t, t["timekey"])
def tickload_dataset_byobj(obj,startdate="",enddate="",source="pkl",url=tick_url):
    import os
    tick_url = url
    fs = os.listdir(tick_url)
    fs_obj = []
    for i in fs:
        if i[:6] == obj:
            fs_obj.append(i)
    fs_obj.sort()
    fs_obj1 = []
    for i in fs_obj:
        date = i[7:15]
        if startdate != "":
            if int(startdate) > int(date):
                continue
        if enddate != "":
            if int(enddate) < int(date):
                continue
        fs_obj1.append(i)
    fs_obj1.sort()
    fs_obj0 = fs_obj1
    ticks=[]
    if source == "csv":
        for i in fs_obj0:
            f_t = tick_url + i
            csv = pd.read_csv(f_t, encoding="gbk")
            csv.rename(columns={'大富翁数据中心_市场代码': "marketcode", '合约代码': "obj", '时间': "timekey", '最新': "LastPrice",
                                '持仓': "position", '增仓': "deltaposition", '成交额': "amont", '成交量': "volume",
                                '开仓': "openvolume", '平仓': "closevolume", '成交类型': "dealfeatures", '方向': "bsflag",
                                '买一价': "BidPrice1", '卖一价': "AskPrice1", '买一量': "BidVolume1", '卖一量': "AskVolume1"},
                       inplace=True)
            csv["timekey"] = csv["timekey"].apply(str2datetime)
            csv["AveragePrice"] = 0
            date = i[7:15]
            csv0=csv.to_dict("records")
            ticks.extend(csv0)
    elif source == "pkl":
        from jili.calc import chan
        for i in fs_obj0:
            f_t = tick_url + i
            date = i[7:15]
            tl = load(f_t)
            ticks.extend(tl)
    return ticks
def barload_dataset_byobj(obj,startdate="",enddate="",source="pkl",url=tick_url):
    bars=[]
    if source!="pkl":
        bars=getkline1m_rb(obj,startdate,enddate,"list")
        from jili.calc import sigma
    else:
        from jili.calc import chan
        f_t = url + obj+"_k1m.pkl"
        bars = load(f_t)
    return bars
def tickengine(obj,call,startdate="",enddate="",show_progress=True,file="pkl",url=tick_url):
    import os
    tick_url=url
    fs=os.listdir(tick_url)
    fs_obj=[]
    for i in fs:
        if i[:6]==obj:
            fs_obj.append(i)
    fs_obj.sort()
    fs_obj1=[]
    for i in fs_obj:
        date=i[7:15]
        if startdate!="":
            if int(startdate)>int(date):
                continue
        if enddate!="":
            if int(enddate)<int(date):
                continue
        fs_obj1.append(i)
    fs_obj1.sort()
    if show_progress:
        fs_obj0=tqdm(fs_obj1)
    else:
        fs_obj0=fs_obj1
    if file=="csv":
        for i in fs_obj0:
            f_t=tick_url+i
            csv=pd.read_csv(f_t,encoding="gbk")
            csv.rename(columns={'大富翁数据中心_市场代码':"marketcode", '合约代码':"obj", '时间':"timekey", '最新':"LastPrice", '持仓':"position", '增仓':"deltaposition", '成交额':"amont" ,'成交量':"volume", '开仓':"openvolume",'平仓':"closevolume", '成交类型':"dealfeatures", '方向':"bsflag", '买一价':"BidPrice1", '卖一价':"AskPrice1", '买一量':"BidVolume1", '卖一量':"AskVolume1"}, inplace = True)
            csv["timekey"]=csv["timekey"].apply(str2datetime)
            csv["AveragePrice"]=0
            date = i[7:15]
            if not show_progress:
                print(i)
            for t in csv.to_dict("records"):
                call(t,date)
    elif file=="pkl":
        from jili.calc import chan
        for i in fs_obj0:
            f_t=tick_url+i
            date = i[7:15]
            tl=load(f_t)
            
            if not show_progress:
                print(i)
            for t in tl:
                t["OpenInterest"]=t["position"]
                call(t, date)
def tickengine_master(obj,call,show_progress=False):
    """
    rb1710:2017-03-23~2017-08-07,4M:20170703~20170815
    rb1801:2017-08-08~2017-11-08,4M:20170704~20171204
    rb1805:2017-11-09~2018-03-27,4M:20170805~20180404
    rb1810:2018-03-28~2018-08-16,4M:20180309~20180827
    rb1901:2018-08-16~2018-11-27,4M:20180725~20181207
    rb1905:2018-11-28~2019-03-08,4M:20181107~20190404
    rb1910:
    :param obj:
    :param call:
    :param show_progress:
    :return:
    """
    from jili.data.obj import getdate_masterobj
    startdate,enddate=getdate_masterobj(obj,"rb")
    startdate=startdate.strftime("%Y%m%d")
    enddate=enddate.strftime("%Y%m%d")
    print(startdate,enddate)
    tickengine(obj,call,startdate,enddate,show_progress)
def getk1mbar_pkl(obj,bar_url0=bar_url):
    name = obj + "_k1m.pkl"
    base_url=os.path.join(os.path.abspath(bar_url0),name)
    return load(base_url)
if __name__=="__main__":
    name = "rb1910" + "_k1m.pkl"
    base_url = os.path.abspath(
        os.path.dirname(os.path.realpath(__file__)) + os.path.sep + "../../data" + os.path.sep + name)
    print(type(base_url),base_url)